{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python3810jvsc74a57bd067e352a81c659ecdd5b9278a63d4406d1f329637477375d76250f1bdeb1757f0",
   "display_name": "Python 3.8.10 64-bit ('base': conda)"
  },
  "metadata": {
   "interpreter": {
    "hash": "df0893f56f349688326838aaeea0de204df53a132722cbd565e54b24a8fec5f6"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "tensor([[-1.5175e+10,  4.5694e-41, -2.3631e+05],\n        [ 4.5694e-41, -2.3157e+05,  4.5694e-41],\n        [-2.3629e+05,  4.5694e-41, -2.3631e+05],\n        [ 4.5694e-41, -2.3465e+05,  4.5694e-41],\n        [-2.3166e+05,  4.5694e-41, -1.5057e+10]])\n"
     ]
    }
   ],
   "source": [
    "x = torch.empty(5, 3)\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "tensor([[0.3576, 0.8707, 0.5974],\n        [0.8823, 0.3511, 0.3371],\n        [0.2579, 0.9353, 0.5274],\n        [0.3314, 0.5905, 0.2655],\n        [0.1135, 0.8254, 0.5610]])\n"
     ]
    }
   ],
   "source": [
    "x = torch.rand(5, 3)\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "tensor([[0.4863, 0.6732, 0.4519],\n",
       "        [0.4576, 0.6536, 0.1937],\n",
       "        [0.3863, 0.1581, 0.7996],\n",
       "        [0.7660, 0.7599, 0.5393],\n",
       "        [0.1626, 0.0293, 0.3881]])"
      ]
     },
     "metadata": {},
     "execution_count": 2
    }
   ],
   "source": [
    "torch.rand(5,3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "tensor([5, 3])\n"
     ]
    }
   ],
   "source": [
    "a = torch.tensor([5,3])\n",
    "type(a)\n",
    "print(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "tensor([[1., 1.],\n        [1., 1.]], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "x = torch.ones(2,2, requires_grad=True)\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "tensor([[3., 3.],\n        [3., 3.]], grad_fn=<AddBackward0>)\n"
     ]
    }
   ],
   "source": [
    "y = x +2\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "z = y*y*3\n",
    "out = z.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "tensor([[27., 27.],\n        [27., 27.]], grad_fn=<MulBackward0>)\n"
     ]
    }
   ],
   "source": [
    "print(z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "tensor(27., grad_fn=<MeanBackward0>)\n"
     ]
    }
   ],
   "source": [
    "print(out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "tensor([[4.5000, 4.5000],\n",
      "        [4.5000, 4.5000]])\n",
      "/home/laine/anaconda3/lib/python3.8/site-packages/torch/autograd/__init__.py:145: UserWarning: CUDA initialization: Unexpected error from cudaGetDeviceCount(). Did you run some cuda functions before calling NumCudaDevices() that might have already set an error? Error 804: forward compatibility was attempted on non supported HW (Triggered internally at  /tmp/pip-req-build-c9mgsz4q/c10/cuda/CUDAFunctions.cpp:109.)\n",
      "  Variable._execution_engine.run_backward(\n"
     ]
    }
   ],
   "source": [
    "out.backward()\n",
    "print(x.grad)"
   ]
  },
  {
   "source": [
    "使用pytorch 构建神经网络"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Net(\n  (conv1): Conv2d(1, 6, kernel_size=(5, 5), stride=(1, 1))\n  (conv2): Conv2d(6, 16, kernel_size=(5, 5), stride=(1, 1))\n  (fc1): Linear(in_features=400, out_features=120, bias=True)\n  (fc2): Linear(in_features=120, out_features=84, bias=True)\n  (fc3): Linear(in_features=84, out_features=10, bias=True)\n)\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Net, self).__init__()\n",
    "        # 输入图像channel：1；输出channel：6；5x5卷积核\n",
    "        self.conv1 = nn.Conv2d(1, 6,5)\n",
    "        self.conv2 = nn.Conv2d(6,16,5)\n",
    "        # an affine operation: y = Wx + b\n",
    "        self.fc1 = nn.Linear(16*5*5, 120)\n",
    "        self.fc2 = nn.Linear(120, 84)\n",
    "        self.fc3 = nn.Linear(84, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        # 2x2 Max pooling\n",
    "        x = F.max_pool2d(F.relu(self.conv1(x)), (2,2))\n",
    "        # 如果是方阵,则可以只使用一个数字进行定义\n",
    "        x = F.max_pool2d(F.relu(self.conv2(x)), 2)\n",
    "        x = x.view(-1, self.num_flat_features(x))\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        x = self.fc3(x)\n",
    "        return x\n",
    "\n",
    "    def num_flat_features(self, x):\n",
    "        size = x.size()[1:] # 除去批处理维度的其他所有维度\n",
    "        num_features = 1\n",
    "        for s in size:\n",
    "            num_features *= s\n",
    "        return num_features\n",
    "\n",
    "net = Net()\n",
    "print(net)"
   ]
  },
  {
   "source": [
    "我们只需要定义 `forward` 函数，`backward`函数会在使用`autograd`时自动定义，`backward`函数用来计算导数。我们可以在 `forward` 函数中使用任何针对张量的操作和计算。\n",
    "\n",
    "一个模型的可学习参数可以通过`net.parameters()`返回"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "10\ntorch.Size([6, 1, 5, 5])\n"
     ]
    }
   ],
   "source": [
    "params = list(net.parameters())\n",
    "print(len(params))\n",
    "print(params[0].size())"
   ]
  },
  {
   "source": [
    "让我们尝试一个随机的 32x32 的输入。注意:这个网络 (LeNet）的期待输入是 32x32 的张量。如果使用 `MNIST` 数据集来训练这个网络，要把图片大小重新调整到 32x32。"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "tensor([[ 0.0983,  0.0543, -0.1042,  0.0900,  0.0576,  0.0027,  0.0492, -0.1156,\n         -0.0637, -0.0093]], grad_fn=<AddmmBackward>)\n"
     ]
    }
   ],
   "source": [
    "input = torch.randn(1,1,32,32, requires_grad=True)  # input之后参与梯度计算，所以需要将 `requires_grad` 设置为True\n",
    "out = net(input)\n",
    "print(out)"
   ]
  },
  {
   "source": [
    "清零所有参数的梯度缓存，然后进行随机梯度的反向传播："
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "net.zero_grad()\n",
    "ix = torch.randn(1,10, requires_grad=True)\n",
    "out.backward(ix, retain_graph=True) # 动态图的设定，图中被多次使用的非叶子节点的梯度值出于节省内存角度会被清空，使用retain_graph可保存梯度内存，但会导致计算变慢"
   ]
  },
  {
   "source": [
    "> 注意：`torch.nn`只支持小批量处理 (`mini-batches`）。整个 `torch.nn` 包只支持小批量样本的输入，不支持单个样本的输入。比如，`nn.Conv2d` 接受一个4维的张量，即 `nSamples` x `nChannels` x `Height` x `Width` 如果是一个单独的样本，只需要使用`input.unsqueeze(0)` 来添加一个“假的”批大小维度。\n",
    "\n",
    "# 复习：\n",
    "`torch.Tensor` - 一个多维数组，支持诸如`backward()`等的自动求导操作，同时也保存了张量的梯度。\n",
    "`nn.Module` ;- 神经网络模块。是一种方便封装参数的方式，具有将参数移动到GPU、导出、加载等功能。\n",
    "`nn.Parameter` ;- 张量的一种，当它作为一个属性分配给一个Module时，它会被自动注册为一个参数。\n",
    "`autograd.Function` - 实现了自动求导前向和反向传播的定义，每个Tensor至少创建一个Function节点，该节点连接到创建Tensor的函数并对其历史进行编码。\n",
    "目前为止，我们讨论了：\n",
    "\n",
    "定义一个神经网络\n",
    "处理输入调用`backward`\n",
    "还剩下：\n",
    "\n",
    "计算损失\n",
    "\n",
    "更新网络权重\n",
    "\n",
    "# 损失函数\n",
    "一个损失函数接受一对(output, target)作为输入，计算一个值来估计网络的输出和目标值相差多少。\n",
    "\n",
    "`nn`包中有很多不同的损失函数。`nn.MSELoss`是比较简单的一种，它计算输出和目标的均方误差(`mean-squared error`）。\n",
    "\n",
    "例如："
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "tensor(0.9465, grad_fn=<MseLossBackward>)\n<MseLossBackward object at 0x7f6002147970>\n"
     ]
    }
   ],
   "source": [
    "output = net(input)\n",
    "target = torch.randn(1,10) # 使用正态生成模拟数据，大小为 1x10\n",
    "#target = target.view(-1, -1) # 使用目标值与数据值尺寸一致\n",
    "criterion = nn.MSELoss()    # mean-squared error\n",
    "loss = criterion(output, target)\n",
    "print(loss)"
   ]
  },
  {
   "source": [
    "现在，如果使用loss的.grad_fn属性跟踪反向传播过程，会看到计算图如下：\n",
    "\n",
    ">input -> conv2d -> relu -> maxpool2d -> conv2d -> relu -> maxpool2d\n",
    ">      -> view -> linear -> relu -> linear -> relu -> linear\n",
    ">      -> MSELoss\n",
    ">      -> loss\n",
    "\n",
    "所以，当我们调用`loss.backward()`，整张图开始关于`loss`微分，图中所有设置了`requires_grad=True`的张量的`.grad`属性累积着梯度张量。\n",
    "\n",
    "为了说明这一点，让我们向后跟踪几步："
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "<MseLossBackward object at 0x7f6001fd68e0>\n<AddmmBackward object at 0x7f6004f65490>\n<AccumulateGrad object at 0x7f60006a5bb0>\n"
     ]
    }
   ],
   "source": [
    "print(loss.grad_fn)\n",
    "print(loss.grad_fn.next_functions[0][0]) # linear\n",
    "print(loss.grad_fn.next_functions[0][0].next_functions[0][0]) # ReLU"
   ]
  },
  {
   "source": [
    "# 反向传播\n",
    "我们只需要调用`loss.backward()`来反向传播误差。我们需要清零现有的梯度，否则梯度将会与已有的梯度累加。\n",
    "\n",
    "现在，我们将调用`loss.backward()`，并查看 `conv1` 层的偏置 (`bias`）在反向传播前后的梯度。"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "conv1.bias.grad before backward\ntensor([0., 0., 0., 0., 0., 0.])\nconv1.bias.grad after backward\ntensor([-0.0148, -0.0085, -0.0037,  0.0179,  0.0043, -0.0078])\n"
     ]
    }
   ],
   "source": [
    "net.zero_grad()\n",
    "print('conv1.bias.grad before backward')\n",
    "print(net.conv1.bias.grad)  # 实际的梯度值\n",
    "loss.backward()\n",
    "print('conv1.bias.grad after backward')\n",
    "print(net.conv1.bias.grad)  # 实际的梯度值"
   ]
  },
  {
   "source": [
    "现在，我们已经见到了如何使用损失函数。\n",
    "\n",
    "稍后阅读神经网络包包含了各种模块和损失函数，这些模块和损失函数构成了深度神经网络的构建模块。完整的文档列表见[这里](https://pytorch.org/docs/stable/nn.html)。现在唯一要学习的是：更新网络的权重\n",
    "# 更新权重\n",
    "最简单的更新规则是随机梯度下降法 (`SGD`）:\n",
    "\n",
    "``` weight = weight - learning_rate * gradient ```\n",
    "\n",
    "我们可以使用简单的 python 代码来实现:"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "for f in net.parameters():\n",
    "    f.data.sub_(f.grad.data * learning_rate)\n",
    "    "
   ]
  },
  {
   "source": [
    "然而，在使用神经网络时，可能希望使用各种不同的更新规则，如 `SGD`、`Nesterov-SGD`、`Adam`、`RMSProp`等。为此，我们构建了一个较小的包 `torch.optim`，它实现了所有的这些方法。使用它很简单："
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.optim as optim\n",
    "# 创建优化器\n",
    "optimizer = optim.SGD(net.parameters(), lr=0.01)\n",
    "# 训练迭代中\n",
    "optimizer.zero_grad()\n",
    "output = net(input)\n",
    "loss = criterion(target, output)\n",
    "loss.backward()\n",
    "optimizer.step() # 更新参数"
   ]
  },
  {
   "source": [
    "注意：观察梯度缓存区是如何使用`optimizer.zero_grad()`手动清零的。这是因为梯度是累加的，正如前面反向传播章节叙述的那样。"
   ],
   "cell_type": "markdown",
   "metadata": {}
  }
 ]
}